    from cProfile import label
    from operator import le
    import re
    import pickle
    from tqdm import tqdm
    import re
    import random

    def step2(file_path):
        pre = set()
        result_sent, sent, record, idx, cixing = [], [], [], [], []
        count = 0
        label = set()
        with open(file_path,'r',encoding='gbk',errors='ignore') as file:
            content = file.readlines()
            for line in tqdm(content):
                ret = re.search('(\<\d+\>)(.+)',line.strip())
                if ret:
                    flag = 0
                    count = ret.group(1)
                    sentence = ret.group(2)
                    pos = sentence.split(' ')
                    fenci = re.sub('\/[a-zA-Z]*','',sentence).split(' ')
                    sent.append(''.join(i for i in fenci))
                    num,i=[],0   
                    for k in fenci:
                        num.append(list(range(i,i+len(k))))
                        i=i+len(k) 
                elif line.strip() == "====PGraph Beg====":
                    continue
                elif line.strip() == "====PGraph End====":
                    if flag == 0:
                        sent.append(num)
                        sent.append(idx)
                        sent.append(record)
                        sent.append(pos)
                        sent.append(cixing)
                        result_sent.append(sent)
                    else:
                        flag = 0
                    sent, record, idx, cixing = [], [], [], []
                elif len(line.strip()) != 0:
                    head = line.strip().split(',')
                    label.add(head[-1])
                    pre.add(head[0])
                    # if head[-1] in ['sbj_HX', 'obj_HX']:
                    record.append((head[0],head[-1],head[2]))
                    x = head[1].strip('()').split(':')
                    y = head[3].strip('()').split(':')
                    if len(set(x)) == 1 :
                        h = [int(i) for i in x[1:]]
                    else:
                        m, n = int(x[0]), int(x[1])
                        h = list(range(m, m+(n-m)+1))
                    if len(set(y)) == 1 :
                        t = [int(i) for i in y[1:]]
                    else:
                        m, n = int(y[0]), int(y[1])
                        t = list(range(m, m+(n-m)+1))
                    if (h in num) and (t in num):
                        idx.append((num.index(h),head[-1],num.index(t)))
                        cixing.append((pos[num.index(h)].split('/')[1],head[-1], pos[num.index(t)].split('/')[1]))
                    else:  #re.search('\/([a-zA-Z]*)',st)
                        flag = 1
        print(label)
        print(len(pre))
        return result_sent
    def get_triple(path):
        with open(path,'r',encoding='utf-8') as file:
            content = file.readlines()
        triple = {}
        for i in range(len(content)):
            if content[i].strip()[0] == '@':
                key = re.split("([A-Z]+)",content[i].strip()[1:])[0]
            elif content[i].strip()[0] == '#':
                rel = content[i].strip()[1:]
                if rel in time:
                    rel = time.get(rel,rel)
                if rel in loc:
                    rel = loc.get(rel,rel)
                if rel == 'Sproc':
                    rel = "Stat"
            else:
                if rel[0] not in ['m','d','r','e'] and rel not in ['Host','Nmod','Int','Seq','Tmod','Desc']:  # 去掉这几个角色
                    word = content[i].strip().split()[0]
                    triple.setdefault(key + '_' + word,set()).add(rel)
        return triple

    def get_obj():
        with open("role_match.txt",'r',encoding='utf-8') as file:
            content = file.readlines()
        # predicate = {"sbj":'Datv', 'obj':'Pat'}
        predicate = {}
        sbj, obj = [], []
        word = ""
        for line in content:
            if line[0] == '@':
                if word != "":
                    if len(sbj) !=0 and len(obj) != 0:
                        predicate[word] = {"sbj":sbj[0], 'obj':obj[0]}
                    if len(sbj) !=0 and len(obj) == 0:
                        predicate[word] = {"sbj":sbj[0]}
                    if len(sbj) ==0 and len(obj) != 0:
                        predicate[word] = {'obj':obj[0]}
                sbj, obj = [], []
                word = line.strip()[1:-3]
            else:
                if len(line.strip().split()) == 3:
                    if line.strip().split()[-1] == 'sbj':
                        sbj.append(line.strip().split()[0])
                    if line.strip().split()[-1] == 'obj':
                        obj.append(line.strip().split()[0])
        return predicate

    '''
    [
    '荆毅定居城市的第一件事，是给三岁的女儿找幼儿园，妻命我去寻。', 
    [[0, 1], [2, 3], [4, 5], [6], [7, 8], [9], [10], [11], [12], [13], [14], [15], [16], [17, 18], [19], [20, 21, 22], [23], [24], [25], [26], [27], [28], [29]], 
    [(8, 'sbj_HX', 6), (8, 'obj_HX', 14), (14, '_BY', 13), (14, 'obj_HX', 15), (18, 'sbj_HX', 17), (18, 'obj_HX', 19), (21, 'sbj_HX', 19)], 
    [('是', 'sbj_HX', '事'), ('是', 'obj_HX', '找'), ('找', '_BY', '女儿'), ('找', 'obj_HX', '幼儿园'), ('命', 'sbj_HX', '妻'), ('命', 'obj_HX', '我'), ('寻', 'sbj_HX', '我')], 
    ['荆毅/nr', '定居/v', '城市/n', '的/u', '第一/m', '件/q', '事/n', '，/w', '是/v', '给/p', '三/m', '岁/q', '的/u', '女儿/n', '找/v', '幼儿园/n', '，/w', '妻/Ng', '命/n', '我/r', '去/v', '寻/v', '。/w']
    [('v','sbj_HX','v'), ('v','sbj_HX','n')。。。]
    ]
    '''
    # {'_BY', 'sbj_HX', 'obj_HX', 'mod_BY'}
    if __name__=="__main__":
        verb = {}
        with open('arg.txt','r',encoding = 'utf-8') as file:
            for line in file:
                head = line.strip().split()
                if len(head) == 3:
                    verb[head[0]] = {"sbj":head[1], 'obj':head[2]}
                if len(head) == 2:
                    verb[head[0]] = {"sbj":head[1]} 
        zy = 0
        flag = 0
        time = {'Tini':'Time','Tfin':'Time','TDur':'Time','Trang':'Time'}
        loc = {'Lini':'Loc','Lfin':'Loc','Lthru':'Loc'}
        sbj = ['Agt', 'Exp', 'Aft', 'Poss','Belg']
        obj = ['Pat', 'Cont', 'Prod', 'Orig', 'Datv', 'Clas']
        label = {'sbj_HX', 'obj_HX', '_BY'}
        other_label = ['GongJu', 'ZhongDian', 'FangShi', 'YiJu', 'CaiLiao', 'LinTi', 'YuanDian']
        path = "../train_data/new_50w_1.out"  # 贵荣跑出的最终jp处理后的数据
        path = "../train_data/manual_data.out"  # 人工标注的数据88,735
        result_sent = step2(path)
        print(len(result_sent),result_sent[:2])
        with open('result_sent.pkl','wb') as out_file:
            pickle.dump(result_sent,out_file)
        result_sent = []
        with open('result_sent.pkl', 'rb') as out_file:
            result_sent = pickle.load(out_file)
        print(result_sent[:1])
        # triple = {'没_本钱': {'Belg'}, '没_下午': {'Time'}, '没_关系': {'Cont', 'Exp', 'mSepa'}}
        triple = get_triple('out6.txt')
        with open('triple.pkl','wb') as out_file:
            pickle.dump(triple,out_file)
        triple = {}
        with open('triple.pkl', 'rb') as out_file:
            triple = pickle.load(out_file)
        print(len(triple), triple['是_事'])
        predicate = get_obj()
        new_result = []
        for item in tqdm(result_sent):
            sent = []
            sent.append(item[0])
            sent.append(item[1])
            idx, record = [], []
            for id, word, cix in zip(item[2],item[3],item[-1]):
                if  id[1][0] == 'd':
                    idx.append((id[0], id[1], {id[1].split('_')[0]}, id[2]))
                    record.append((word[0], word[1], {word[1].split('_')[0]}, word[2]))
                    continue
                if cix[2] == 'v':
                    if id[1] == '_BY' or id[1].split('_')[1] == 'BY1':
                        continue
                    elif id[1].split('_')[0] in ['GongJu', 'ZhongDian', 'FangShi', 'YiJu', 'CaiLiao', 'LinTi', 'YuanDian']:
                        idx.append((id[0], id[1], {id[1].split('_')[0]}, id[2]))
                        record.append((word[0], word[1], {word[1].split('_')[0]}, word[2]))
                        continue
                elif id[1].split('_')[0] in ['GongJu', 'ZhongDian', 'FangShi', 'YiJu', 'CaiLiao', 'LinTi', 'YuanDian']:
                    idx.append((id[0], id[1], {id[1].split('_')[0]}, id[2]))
                    record.append((word[0], word[1], {id[1].split('_')[0]}, word[2]))
                    continue
                elif word[0] + '_' + word[2] in triple:  # 如果搭配在依存角色数据中匹配到了角色
                    if len(triple[word[0] + '_' + word[2]]) > 1 and id[1] in ['sbj_HX','sbj_BY','obj_HX','obj_BY']:
                        if id[1] in ['sbj_HX','sbj_BY']:
                            box = []
                            for can in sbj:
                                if can in triple[word[0] + '_' + word[2]]:
                                    box.append(can)
                            if len(box) > 1:
                                idx.append((id[0], id[1], {'Agt'}, id[2]))
                                record.append((word[0], word[1], {'Agt'}, word[2]))
                            elif len(box) == 1:
                                idx.append((id[0], id[1], {box[0]}, id[2]))
                                record.append((word[0], word[1], {box[0]}, word[2]))
                            elif cix[2] in ['n', 'r', 'nr', 'ns', 'nt', 'nz', 'vn', 'Ng', 's', 'an', 'i', 'j']:
                                idx.append((id[0], id[1], {'Agt'}, id[2]))
                                record.append((word[0], word[1],{'Agt'}, word[2]))
                            elif  'Freq' in triple[word[0] + '_' + word[2]] and 'Time' in triple[word[0] + '_' + word[2]] and word[2] in ['年','天','月']:
                                idx.append((id[0], id[1], {'Time'}, id[2]))
                                record.append((word[0], word[1], {'Time'}, word[2]))
                            elif  'Qp' in triple[word[0] + '_' + word[2]] and 'Time' in triple[word[0] + '_' + word[2]] and word[2] in ['年','天','月']:
                                idx.append((id[0], id[1], {'Time'}, id[2]))
                                record.append((word[0], word[1], {'Time'}, word[2]))
                            elif ('Freq' in triple[word[0] + '_' + word[2]] or 'Qp' in triple[word[0] + '_' + word[2]]) and 'Time' not in triple[word[0] + '_' + word[2]]:
                                idx.append((id[0], id[1], {'Quan'}, id[2]))
                                record.append((word[0], word[1], {'Quan'}, word[2]))
                            else:
                                idx.append((id[0], id[1], triple[word[0] + '_' + word[2]], id[2]))
                                record.append((word[0], word[1],triple[word[0] + '_' + word[2]], word[2]))
                        elif id[1] in ['obj_HX','obj_BY']:
                            box = []
                            for can in obj:
                                if can in triple[word[0] + '_' + word[2]]:
                                    box.append(can)
                            if len(box) > 1:
                                idx.append((id[0], id[1], {'Pat'}, id[2]))
                                record.append((word[0], word[1], {'Pat'}, word[2]))
                            elif len(box) == 1:
                                idx.append((id[0], id[1], {box[0]}, id[2]))
                                record.append((word[0], word[1], {box[0]}, word[2]))
                            elif cix[2] in ['n', 'r', 'nr', 'ns', 'nt', 'nz', 'vn', 'Ng', 's', 'an', 'i', 'j']:
                                idx.append((id[0], id[1], {'Pat'}, id[2]))
                                record.append((word[0], word[1],{'Pat'}, word[2]))
                            elif  'Freq' in triple[word[0] + '_' + word[2]] and 'Time' in triple[word[0] + '_' + word[2]] and word[2] in ['年','天','月']:
                                idx.append((id[0], id[1], {'Time'}, id[2]))
                                record.append((word[0], word[1], {'Time'}, word[2]))
                            elif  'Qp' in triple[word[0] + '_' + word[2]] and 'Time' in triple[word[0] + '_' + word[2]] and word[2] in ['年','天','月']:
                                idx.append((id[0], id[1], {'Time'}, id[2]))
                                record.append((word[0], word[1], {'Time'}, word[2]))
                            elif ('Freq' in triple[word[0] + '_' + word[2]] or 'Qp' in triple[word[0] + '_' + word[2]]) and 'Time' not in triple[word[0] + '_' + word[2]]:
                                idx.append((id[0], id[1], {'Quan'}, id[2]))
                                record.append((word[0], word[1], {'Quan'}, word[2]))
                            else:
                                idx.append((id[0], id[1], triple[word[0] + '_' + word[2]], id[2]))
                                record.append((word[0], word[1], triple[word[0] + '_' + word[2]], word[2]))
                    else:  # 1. 匹配上多个role但是label!=obj,sbj 2. 匹配上一个role但是label=obj,sbj, 3. 匹配上一个role但是label=BY
                        if len(triple[word[0] + '_' + word[2]]) == 1 and id[1] in ['sbj_HX', 'obj_HX','sbj_BY','obj_BY']:
                            if 'Freq' in triple[word[0] + '_' + word[2]] or 'Qp' in triple[word[0] + '_' + word[2]]:
                                idx.append((id[0], id[1], {'Quan'}, id[2]))
                                record.append((word[0], word[1], {'Quan'}, word[2]))
                            elif 'Comp' in triple[word[0] + '_' + word[2]] and id[1] in ['sbj_HX','sbj_BY']:
                                idx.append((id[0], id[1], {'Exp'}, id[2]))
                                record.append((word[0], word[1], {'Exp'}, word[2]))
                            else:
                                idx.append((id[0], id[1], triple[word[0] + '_' + word[2]], id[2]))
                                record.append((word[0], word[1], triple[word[0] + '_' + word[2]], word[2]))
                        elif len(triple[word[0] + '_' + word[2]]) == 1 and 'Comp' in triple[word[0] + '_' + word[2]] and id[1] == "_BY":
                            idx.append((id[0], id[1], {}, id[2]))
                            record.append((word[0], word[1],  {}, word[2]))
                        elif len(triple[word[0] + '_' + word[2]]) > 1 and 'Comp' in triple[word[0] + '_' + word[2]] and id[1] == "_BY":
                            triple[word[0] + '_' + word[2]].remove('Comp')
                            idx.append((id[0], id[1], triple[word[0] + '_' + word[2]], id[2]))
                            record.append((word[0], word[1],  triple[word[0] + '_' + word[2]], word[2]))
                        else:
                            idx.append((id[0], id[1], triple[word[0] + '_' + word[2]], id[2]))
                            record.append((word[0], word[1], triple[word[0] + '_' + word[2]], word[2]))
                else:  # 找不到role, 如果是sbj&obj,按照统计的高频角色给定
                    if word[1] in ['sbj_HX', 'obj_HX','sbj_BY','obj_BY']:
                        if word[0] in predicate and word[1].split('_')[0] in predicate[word[0]]:
                            idx.append((id[0], id[1], {predicate[word[0]][word[1].split('_')[0]]}, id[2]))
                            record.append((word[0], word[1], {predicate[word[0]][id[1].split('_')[0]]}, word[2]))
                            zy += 1
                        elif word[0] in verb and word[1].split('_')[0] in verb[word[0]]:
                            idx.append((id[0], id[1], {verb[word[0]][word[1].split('_')[0]]}, id[2]))
                            record.append((word[0], word[1], {verb[word[0]][id[1].split('_')[0]]}, word[2]))
                        elif len(word[0]) == 4 or  cix[0] == 'a':  # 如果是四字成语和形容词谓词，则论元角色为Exp
                            idx.append((id[0], id[1], {"Exp"}, id[2]))
                            record.append((word[0], word[1], {"Exp"}, word[2]))
                        elif cix[2] in ['s','ns','nt']:
                            idx.append((id[0], id[1], {'Loc'}, id[2]))
                            record.append((word[0], word[1], {'Loc'}, word[2]))
                        else:
                            idx.append((id[0], id[1], {}, id[2]))
                            record.append((word[0], word[1], {}, word[2]))
                    elif cix[2] in ['s','ns','nt']:
                        idx.append((id[0], id[1], {'Loc'}, id[2]))
                        record.append((word[0], word[1], {'Loc'}, word[2]))
                    else: 
                        idx.append((id[0], id[1], {}, id[2]))
                        record.append((word[0], word[1], {}, word[2]))
            sent.append(idx)
            sent.append(record)
            sent.append(item[-2])
            sent.append(item[-1])
            new_result.append(sent)
        print("len(result_sent): ", len(result_sent))
        print("len(new_result): ", len(new_result)) 
        flag = 1
        output = {}
        print('zyu:', zy)
        catch, C = 0, 0
        flag , ff, only = 0, 0, 0
        least = 0
        ## 全部的转换了角色之后的
        value_sent = []
        with open('50w_multi_role_3.txt','w',encoding='utf-8') as out:
            for item in new_result:
                C += 1
                out.write('@' + ' '.join(i for i in item[-2]) + '\n')
                out.write("====PGraph Beg====\n")
                for key in item[3]:  # ('有', 'obj_HX', {'exp','agt','clas'},'幼儿园')
                    if len(key[2]) == 1:
                        pass
                    else:
                        only = 1
                    if len(key[2]) == 0:
                        flag = 1
                    else:
                        ff =1
                    out.write(key[0] + ' [' + ' '.join(i for i in key[2]) + '] ' + key[3] + ' ' + key[1] + '\n')
                if flag == 0:
                    catch += 1
                else:
                    flag = 0
                if ff == 1:
                    least += 1
                    ff = 0
                if only == 0:
                    value_sent.append(item)
                else:
                    only = 0
                out.write("====PGraph End====\n\n\n")            
        print(C, catch, least, catch/C, least/C)
        
        
        ## 统计有效句子(每个谓词-论元搭配都有一个唯一的角色的情况)
        print(len(value_sent))
        
        ### 处理出 train data
        labeled = {}
        final_data = []
        for item in tqdm(new_result):
            sent = []
            sent.append(item[0])
            sent.append(item[1])
            idx, record = [], []   # (6, 'obj_HX', {'exp','agt','clas'}, 9)
            for id, word in zip(item[2],item[3]):   # ('有', 'obj_HX', {'exp','agt','clas'},'幼儿园')
                if len(id[2]) != 1 or list(id[2])[0] == 'Qp' or list(id[2])[0] == 'Comp':  #过滤掉空的角色和多个角色的搭配
                    continue
                elif list(id[2])[0] == 'Orig':
                    idx.append((id[0], id[1], 'YuanDian', id[3]))
                    record.append((word[0], word[1], 'YuanDian', word[3]))
                    labeled['YuanDian'] = labeled.get('YuanDian',0) +1
                else:
                    idx.append((id[0], id[1], list(id[2])[0], id[3]))
                    record.append((word[0], word[1], list(id[2])[0], word[3]))
                    labeled[list(word[2])[0]] = labeled.get(list(word[2])[0],0) +1
            sent.append(idx)
            sent.append(record)
            sent.append(item[-2])
            if len(sent[-2]) != 0:
                final_data.append(sent)
                
        with open('final_data.pkl','wb') as out_file:
            pickle.dump(final_data,out_file)
        print(len(final_data),final_data[0])  # 212784
        print(len(labeled))
        vd = sorted(labeled.items(),key=lambda t:t[1])
        print (vd)
        list_info = [25, 48, 326, 367, 425, 443, 518, 538, 630, 799, 951, 1008, 1100, 1273, 1288, 1318, 1391, 1408, 1574, 1718, 1768, 1838, 1896, 1947, 1951, 1999, 2394, 2456, 2510, 2683, 2788, 2809, 2871, 3034, 3043, 3123, 3126, 3262, 3334, 3463, 3575, 3714, 3756, 3783, 3825, 3845, 3852, 3904, 4029, 4057, 4207, 4240, 4303, 4330, 4357, 4572, 4733, 4824, 4884, 4959, 5044, 5086, 5104, 5171, 5421, 5447, 5511, 5548, 5583, 5609, 5862, 6367, 6412, 6483, 6600, 6641, 6657, 6730, 6748, 6824, 6942, 6947, 6982, 7074, 7151, 7164, 7167, 7307, 7365, 7414, 7470, 7495, 7511, 7642, 7767, 7770, 8113, 8279, 8574, 8576, 8799, 8905, 8936, 8979, 8989, 9099, 9152, 9250, 9308, 9326, 9367, 9400, 9488, 9506, 9701, 9751, 9762, 9776, 9813, 9883, 9896, 9956, 9993, 10101, 10221, 10227, 10303, 10378, 10441, 10452, 10473, 10546, 10591, 10612, 10759, 10775, 10820, 10999, 11067, 11069, 11093, 11138, 11144, 11215, 11363, 11371, 11405, 11424, 11485, 11504, 11970, 12035, 12178, 12283, 12562, 12632, 12736, 12738, 12835, 12890, 12951, 13152, 13158, 13167, 13389, 13454, 13554, 13563, 13896, 13978, 14100, 14145, 14156, 14187, 14238, 14252, 14283, 14325, 14555, 14692, 14770, 14792, 14806, 15175, 15449, 15593, 15637, 15689, 15720, 15840, 15867, 15917, 16000, 16051, 16069, 16412, 16482, 16600, 16603, 16705, 16844, 16925, 16928, 16937, 16938, 16958, 17075, 17191, 17196, 17319, 17381, 17481, 17516, 17642, 17728, 17855, 17857, 17988, 18052, 18059, 18092, 18200, 18438, 18537, 18589, 18699, 18740, 18891, 18991, 19006, 19062, 19161, 19222, 19305, 19394, 19501, 19683, 19794, 20053, 20310, 20500, 20528, 20555, 20654, 20732, 20784, 21001, 21028, 21251, 21296, 21319, 21344, 21549, 21870, 22124, 22178, 22301, 22449, 22461, 22641, 22660, 22900, 22992, 23006, 23032, 23188, 23215, 23311, 23491, 23551, 23601, 23659, 23834, 23838, 23946, 23992, 24046, 24144, 24195, 24258, 24325, 24417, 24540, 24557, 24680, 24957, 25039, 25153, 25225, 25543, 25566, 25648, 25660, 25706, 25723, 25808, 25811, 25881, 25943, 25953, 25991, 26042, 26280, 26472, 26503, 26596, 26750, 26867, 26903, 26976, 27025, 27033, 27054, 27062, 27074, 27164, 27212, 27305, 27326, 27419, 27575, 27604, 27626, 27633, 27673, 27707, 27800, 27844, 27866, 27879, 28047, 28059, 28061, 28117, 28170, 28196, 28247, 28248, 28305, 28400, 28427, 28460, 28541, 28550, 28629, 28743, 28909, 28959, 29054, 29070, 29082, 29087, 29109, 29260, 29360, 29370, 29373, 29394, 29427, 29493, 29645, 29680, 29738, 29764, 29908, 29974, 30021, 30042, 30049, 30158, 30178, 30229, 30411, 30465, 30488, 30656, 30733, 30969, 30974, 30985, 30992, 31049, 31094, 31267, 31353, 31487, 31521, 31740, 31747, 31790, 31854, 31897, 32137, 32183, 32326, 32348, 32402, 32415, 32502, 32659, 32678, 32705, 32709, 32762, 32832, 33136, 33197, 33232, 33403, 33521, 33553, 33757, 33787, 33978, 33996, 33998, 34130, 34204, 34231, 34237, 34284, 34303, 34453, 34639, 34648, 34750, 34753, 34764, 34820, 34929, 35197, 35201, 35257, 35346, 35457, 35550, 35903, 35978, 36217, 36291, 36384, 36407, 36504, 36512, 36722, 36816, 36859, 36954, 36996, 36998, 37288, 37339, 37359, 37390, 37392, 37466, 37491, 37557, 37701, 37746, 37790, 37935, 38014, 38114, 38171, 38196, 38455, 38489, 38702, 38705, 38900, 38958, 38971, 38981, 38990, 39056, 39062, 39203, 39261, 39354, 39381, 39412, 39522, 39594, 39651, 39879, 40026, 40030, 40035, 40245, 40263, 40298, 40338, 40353, 40491, 40505, 40514, 40565, 40799, 40822, 40845, 40858, 40944, 40997, 41221, 41488, 41567, 41596, 41799, 41862, 42008, 42022, 42202, 42306, 42564, 42822, 43226, 43355, 43366, 43414, 43464, 43545, 43560, 43576, 43674, 43805, 43928, 43997, 44085, 44267, 44270, 44310, 44330, 44446, 44768, 44789, 44819, 44927, 45125, 45142, 45189, 45211, 45216, 45335, 45464, 45484, 45537, 45597, 45662, 45672, 45676, 45950, 45969, 46079, 46243, 46323, 46391, 46730, 46804, 47004, 47038, 47280, 47423, 47539, 47572, 47720, 47892, 47925, 48322, 48402, 48525, 48618, 48647, 48664, 48729, 48787, 48983, 49059, 49219, 49255, 49339, 49439, 49593, 49732, 49754, 49811, 49952, 50009, 50133, 50148, 50152, 50220, 50287, 50359, 50405, 50479, 50507, 50508, 50570, 50721, 50789, 50955, 51021, 51064, 51240, 51246, 51340, 51401, 51594, 51614, 51740, 51741, 51816, 51892, 51895, 51910, 51926, 51936, 51965, 52070, 52150, 52294, 52369, 52533, 52694, 52759, 52864, 52981, 53298, 53407, 53533, 53600, 53647, 53845, 54008, 54036, 54097, 54320, 54416, 54491, 54513, 54614, 54633, 54963, 55008, 55014, 55036, 55150, 55186, 55217, 55335, 55404, 55405, 55475, 55573, 55605, 55662, 55667, 55861, 55865, 55867, 55869, 55966, 56007, 56033, 56045, 56123, 56414, 56449, 56606, 56675, 56920, 57147, 57272, 57278, 57344, 57352, 57468, 57525, 57898, 58026, 58318, 58324, 58433, 58757, 58842, 58865, 58884, 58910, 59256, 59281, 59339, 59425, 59434, 59455, 59501, 59584, 59653, 59656, 59724, 59755, 59792, 59806, 59979, 60078, 60197, 60414, 60657, 60701, 60723, 60847, 60852, 60957, 61056, 61103, 61172, 61210, 61220, 61305, 61347, 61349, 61356, 61383, 61605, 61620, 61868, 61890, 61897, 62024, 62078, 62160, 62210, 62213, 62465, 62603, 62625, 62736, 62742, 62762, 62830, 62884, 62906, 63043, 63200, 63214, 63227, 63251, 63259, 63299, 63409, 63588, 63886, 63902, 63942, 63945, 63947, 64102, 64170, 64174, 64212, 64334, 64383, 64486, 64531, 64658, 64736, 64780, 64834, 64951, 64982, 65333, 65382, 65453, 65510, 65524, 65656, 65841, 65923, 65970, 66014, 66034, 66313, 66608, 66986, 66992, 67025, 67036, 67096, 67125, 67302, 67382, 67461, 67559, 67577, 67665, 67797, 67804, 67807, 68000, 68074, 68084, 68091, 68154, 68226, 68241, 68625, 68630, 68796, 68818, 68914, 68951, 69290, 69478, 69606, 69689, 69705, 69732, 69838, 69920, 70164, 70224, 70327, 70399, 70420, 70503, 70619, 70623, 70853, 70858, 70983, 71041, 71044, 71116, 71174, 71228, 71361, 71371, 71547, 71623, 71721, 71726, 71789, 71851, 71856, 71935, 72044, 72194, 72198, 72237, 72288, 72289, 72352, 72387, 72428, 72584, 72605, 72630, 72637, 72814, 72860, 72870, 73223, 73263, 73276, 73401, 73514, 73568, 73609, 73718, 73759, 73763, 73775, 73900, 73975, 74153, 74198, 74424, 74439, 74630, 74654, 74765, 74789, 74883, 74929, 75198, 75306, 75354, 75686, 75741, 75755, 75767, 75819, 75885, 75916, 76075, 76086, 76107, 76289, 76318, 76439, 76472, 76511, 76557, 76558, 76583, 76647, 76825, 76961, 77140, 77192, 77196, 77343, 77415, 77465, 77480, 77550, 77555, 77730, 77750, 77805, 77862, 77901, 77935, 78119, 78151, 78240, 78442, 78451, 78485, 78540, 78605, 78710, 78775, 78918, 78953, 78998, 79038, 79176, 79208, 79357, 79399, 79488, 79620, 79638, 79772, 79796, 79830, 79893, 79929, 80115, 80171, 80194, 80370, 80409, 80441, 80497, 80532, 80542, 80594, 80610, 80640, 80840, 80870, 80889, 80917, 80976, 81073, 81171, 81184, 81251, 81320, 81474, 81648, 81657, 81682, 81691, 81741, 81853, 82032, 82033, 82048, 82079, 82183, 82191, 82274, 82312, 82432, 82454, 82512, 82587, 82658, 82669, 82728, 82801, 82887, 82933, 82945, 82947, 83030, 83118]
        gold_data = []
        train_val = []
        for i in range(len(final_data)):
            if i in list_info:
                gold_data.append(final_data[i])
            else:
                train_val.append(final_data[i])
        print(len(gold_data),len(train_val))  # 1000, 82182
        new_gold = []
        for item in gold_data:
            N, po, t = [], [], []
            for i in range(len(item[-1])):
                po.append(str(i) + '_' + item[-1][i])
            N.append(po)
            for a,b in zip(item[2],item[3]):
                t.append((str(a[0]) + '_' + b[0], a[1], a[2], str(a[3]) + '_' + b[3]))
            N.append(t)
            new_gold.append(N)
        with open('gold_data.txt','w',encoding = 'utf-8') as file:
            for item in new_gold:
                file.write('@' + ' '.join(i for i in item[0]) + '\n')
                file.write("====PGraph Beg====\n")
                for key in item[1]:  # ('是', 'sbj_HX', 'Exp', '事')
                    file.write(key[0] + ' ' + key[1] + ' ' + key[2] + ' ' + key[3] + '\n')
                file.write("====PGraph End====\n\n\n")
        
        with open('../shuffle_manual_role/gold_unmanual.pkl','wb') as out_file:
            pickle.dump(gold_data,out_file)
        
        print(len(labeled))
        vd = sorted(labeled.items(),key=lambda t:t[1],reverse=True)
        print (vd)
        train_f = '../shuffle_manual_role/train_data.pkl'
        val = '../shuffle_manual_role/val_data.pkl'
        test = '../shuffle_manual_role/test_data.pkl'
        
        from sklearn.model_selection import train_test_split
        # 先将1.训练集，2.验证集+测试集，按照8：2进行随机划分
        X_train, X_validate_test= train_test_split(train_val, test_size = 0.025, random_state = 42)
        # 再将1.验证集，2.测试集，按照1：1进行随机划分
        X_validate, X_test, = train_test_split(X_validate_test, test_size = 0.5, random_state = 42)
    
        print("len(X_train),len(X_validate),len(X_test):", len(X_train),len(X_validate),len(X_test))
        with open(train_f,'wb') as out_file:  # 81182
            pickle.dump(X_train,out_file) 
        with open(val,'wb') as out_file:
            pickle.dump(X_validate,out_file) 
        with open(test,'wb') as out_file:
            pickle.dump(X_test,out_file)






















    '''
    make_data
    batch[3] :'抓住这一环，就为提高产品质量打下了基础。'
    batch[4] :[[0, 1], [2], [3], [4], [5], [6], [7], [8, 9]] 
    batch[5] :[(8, 'sbj', 6), (8, 'obj', 14)]
    batch[6] :[('抓住', 'obj', '环'), ('打下', 'obj', '基础')]
    batch[7] :['抓住/v', '这/r', '一/m', '环/n', '，/w', '就/d', '为/p', '提高/v', '产品/n', '质量/n', '打下/v', '了/y', '基础/n', '。/w']
    [('v', 'sbj_HX', 'n'), ('v', 'obj_HX', 'v'), ('v', '_BY', 'n'), ('v', 'obj_HX', 'n'), ('n', 'sbj_HX', 'Ng'), ('n', 'obj_HX', 'r'), ('v', 'sbj_HX', 'r')]
    '''
                
        
        

